import csv
import requests
from bs4 import BeautifulSoup


def get_html(url):
    head = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36'}
    try:
        r = requests.get(url=url, headers=head)
        r.encoding = r.apparent_encoding
        r.raise_for_status()
        return r.text
    except Exception as e:
        print(e)


def parser(html):
    soup = BeautifulSoup(html, 'lxml')
    out_list = []
    print(soup.select('#colR ul>li'))
    for row in soup.select('#colR ul>li'):
        row_list = [row.text.replace(' ', '').replace("\n", ""),
                    row.select("a")[0].attrs["href"].replace(' ', '').replace("\n", "")]
        out_list.append(row_list)
    return out_list


def save_csv(path, item):
    with open(path, 'a+', encoding='utf-8', newline='')as f:
        csv_write = csv.writer(f)
        csv_write.writerows(item)


if __name__ == '__main__':
    for i in range(1,6):
        if i==1:
            url = 'https://www.cqie.edu.cn/html/2/xydt/Index.shtml'
        else:
            url = f'https://www.cqie.edu.cn/html/2/xydt/List_{i}.shtml'
        html = get_html(url)
        out_list = parser(html)
        path = 'd://校园动态.csv'
        save_csv(path, out_list)